Applied Data Science Capstone - Week 3¶
Segmenting and Clustering Neighborhoods in Toronto¶
Contents¶
Scrape Wiki Page for Toronto's Postal Codes¶
Geocode¶
Additional Data for Toronto¶
Explore Neighborhoods Cluster Data Maps¶
In [ ]:
In [ ]:
#### Tip for links: [Link to the destination](#the_destination) followed by <a id='K-Means_Clustering'></a>
In [ ]:
In [1]:
import requests
# http requests & response used with Foursquare API
import json # library to handle JSON files
import pandas as pd
from pandas.io.json import json_normalize # tranform JSON file into a pandas dataframe
# Foursquare results will be in JSON and then flattened and stored in a Pandas Dataframe
In [2]:
from bs4 import BeautifulSoup
# Beautiful Soup is a Python library for pulling data out of HTML and XML files.
In [3]:
import numpy as np # library to handle data in a vectorized manner
#!conda install -c conda-forge geopy --yes # uncomment this line if you haven't completed the Foursquare API lab
from geopy.geocoders import Nominatim # convert an address into latitude and longitude values
# Geopy can calculate geodesic distance between two points using the geodesic distance or the great-circle distance,
# with a default of the geodesic distance available as the function geopy.distance.distance.
from geopy.distance import geodesic
from geopy.distance import great_circle
import scipy
from scipy.spatial import distance
# Matplotlib and associated plotting modules
%matplotlib inline
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import matplotlib.colors as colors
# import k-means for clustering
from sklearn.cluster import KMeans
#!conda install -c conda-forge folium=0.5.0 --yes # uncomment this line if you haven't completed the Foursquare API lab
import folium # map rendering library
print('Libraries imported.')
In [4]:
# folium should be version 0.11.0
print(folium.__version__)
In [5]:
import pgeocode
# pgeocode is a Python library for high performance off-line querying of GPS coordinates, region name and municipality
# name from postal codes. Distances between postal codes as well as general distance queries are also supported.
# The used GeoNames database includes postal codes for 83 countries.
# Currently, only queries within the same country are supported.
# For additional documentation see pgeocode.readthedocs.io.
In [6]:
pgeocode.__version__
Out[6]:
In [7]:
import geopandas as gpd
import geojson
In [8]:
# Wide Display
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))
In [ ]:
In [ ]:
In [9]:
# Files saved:
# 'North_York_Starbucks_20200616b.pkl'
# 'Toronto_Downtown_Starbucks_20200616b.pkl'
# 'Toronto_Scarborough_Starbucks_20200616b.pkl'
# 'Toronto_West_York_Starbucks_20200616b.pkl'
# df.to_pickle(file_name) # where to save it, usually as a .pkl
# df = pd.read_pickle(file_name)
In [ ]:
Define Foursquare Credentials and Version¶
In [10]:
CLIENT_ID = '1O1KN544PAGAVX5SIEEO4SINNIYJTELH1YMHPSOPLESWN1VF' # your Foursquare ID
CLIENT_SECRET = '3VB4HIYI1IYA0SYA5L3TLAJM0P4KFGOZ45FRMYMSGXCR4FTV' # your Foursquare Secret
VERSION = '20200514' # Foursquare API version
print('Your credentails:')
print('CLIENT_ID: ' + CLIENT_ID)
print('CLIENT_SECRET:' + CLIENT_SECRET)
In [11]:
# function that extracts the category of the venue from Foursquare results
def get_categories(row):
try:
categories_list = row['categories']
except:
categories_list = row['venue.categories']
if len(categories_list) == 0:
return None
else:
return categories_list[0]['name']
In [ ]:
Wikipedia page used to gather data that is in a table of postal codes and then transform the data into a pandas dataframe¶
In [13]:
# https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M
# Metropolitan Toronto: M 102 Postal Codes for Toronto; 140 Neighbourhoods
url = 'http://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M'
In [14]:
response = requests.get(url)
In [15]:
# Beautiful Soup supports the HTML parser included in Python’s standard library,
# but it also supports a number of third-party Python parsers. One is the lxml parser.
soup = BeautifulSoup(response.text, 'lxml')
In [16]:
# WORKS! For CSS Class wikitable sortable jquery
# If you want to search for tags that match two or more CSS classes, you should use a CSS selector:
# soup.select("table.wikitable.sortable")
In [17]:
postal_table = soup.select("table.wikitable.sortable")
In [18]:
# Use Pandas to read html table into a dataaframe.
postal_df = pd.read_html(str(postal_table))[0]
In [19]:
#PostalCode-Borough-Neighborhood-Dataframe
In [20]:
postal_df
Out[20]:
In [21]:
postal_df.shape
Out[21]:
In [22]:
# 77 rows have Borough values 'Not Assigned'
# 9 Boroughs within Toronto
# Mississauga is a city outside of Toronto but has one Toronto assigned postal code, M7R for its processing centre.
postal_df.Borough.value_counts()
Out[22]:
In [23]:
postal_df[postal_df['Borough'].str.contains('Not assigned')]
Out[23]:
In [24]:
# Test if any Borough values do not match 'Not assigned', and there are not any.
postal_df[~postal_df['Borough'].str.contains('Not assigned')]
Out[24]:
Create new dataframe of rows where 'Borough' column is NOT string 'Not assigned'¶
In [25]:
# Create new dataframe of rows where 'Borough' column is NOT string 'Not assigned'
assigned_postal_df = postal_df[~postal_df['Borough'].str.contains('Not assigned')]
In [26]:
assigned_postal_df.shape
Out[26]:
In [27]:
assigned_postal_df
Out[27]:
In [28]:
sorted_postal_df = assigned_postal_df.sort_values('Borough')
In [29]:
sorted_postal_df.head(50)
Out[29]:
In [30]:
sorted_postal_df.iloc[49:99]
Out[30]:
In [31]:
sorted_postal_df.iloc[98:104]
Out[31]:
In [ ]:
In [32]:
# Cleaned-Sorted-PostalCode-Dataframe
In [33]:
sorted_postal_df.head(10)
Out[33]:
In [34]:
sorted_postal_df.shape
Out[34]:
In [35]:
# 9 Boroughs listed in table
In [36]:
# NOTE: Some Postal Codes have multiple Neighborhoods assigned.
In [37]:
# City of Toronto’s 140 neighbourhoods displayed by neighbourhood number
# https://www.toronto.ca/city-government/data-research-maps/neighbourhoods-communities/neighbourhood-profiles/
In [38]:
# A forward sortation area (FSA) is a geographical region in which all postal codes start with the same three characters.
# The first letter of an FSA code denotes a particular "postal district", "M" for Toronto.
# The digit identifies the FSA as urban or rural. A zero indicates a wide-area rural region; all other digits indicate urban areas.
# Note: There are no rural FSAs in Toronto, hence no postal codes should start with M0.
# The second letter represents a specific rural region, an entire medium-sized city, or a section of a major metropolitan area.
# Special example is NORTH POLE H0H 0H0
# Toronto has 180 Postal Codes reserved but 77 are Not Assigned, leaving 103 Assigned Postal Codes.
In [ ]:
Create a list of Postal Codes from the dataframe¶
In [39]:
PC_array = assigned_postal_df['Postal Code'].to_numpy()
In [40]:
PC_array
Out[40]:
In [ ]:
In [ ]:
Geocoding Postal Codes¶
In [41]:
nomi = pgeocode.Nominatim('CA')
nomi.query_postal_code("M1W")
Out[41]:
In [42]:
# Create a new dataframe from query of postal codes for lat & lng
postal_ll_df = nomi.query_postal_code(PC_array)
postal_ll_df.dtypes
Out[42]:
In [43]:
type(postal_ll_df)
Out[43]:
In [44]:
postal_ll_df
Out[44]:
In [45]:
postal_ll_df.columns
Out[45]:
In [46]:
# filter columns
filtered_columns = ['postal_code', 'latitude', 'longitude']
postal_ll_df = postal_ll_df.loc[:, filtered_columns]
In [47]:
postal_ll_df
Out[47]:
In [48]:
assigned_postal_df.columns
Out[48]:
In [49]:
postal_ll_df.columns
Out[49]:
In [50]:
# Edit column so key on column name matches for merge
In [51]:
edit_postal_df = assigned_postal_df.rename(columns={'Postal Code': 'postal_code'})
In [52]:
edit_postal_df.columns
Out[52]:
Merge the two dataframes with common postal code values to assign latitude & longitude to borough/ neighborhoods.¶
In [55]:
# result = pd.merge(left, right, on='key')
In [56]:
postal_neighbourhood_ll = pd.merge(edit_postal_df, postal_ll_df, on='postal_code')
In [57]:
postal_neighbourhood_ll
Out[57]:
Resulting Table of Postal Codes, Neighborhoods, Latitude & Longitude¶
In [52]:
# df.to_pickle(file_name)
# where to save it, usually as a .pkl
# df = pd.read_pickle(file_name)
postal_neighbourhood_ll.to_pickle('postal_neighbourhood_lat_lng_20200629.pkl')
In [ ]:
Load a new dataframe from Open Toronto Neighbourhoods csv file¶
In [58]:
# Download https://open.toronto.ca/dataset/neighbourhoods/
# https://ckan0.cf.opendata.inter.prod-toronto.ca/download_resource/a083c865-6d60-4d1d-b6c6-b0c8a85f9c15?format=geojson&projection=4326
# neighbourhood-profiles-2016-csv https://ckan0.cf.opendata.inter.prod-toronto.ca/download_resource/ef0239b1-832b-4d0b-a1f3-4153e53b189e?format=csv
# Transpose Rows / Columns .T
# toronto_profiles = pd.read_csv('neighbourhood-profiles-2016-csv.csv', index_col=0, header=None).T
# toronto_profiles = pd.read_csv('neighbourhood-profiles-2016-csv.csv', index_col=0, header=None)
# toronto_profiles = pd.read_csv('cleaned_neighbourhood-profiles-2016-csv_UTF8.csv', index_col=0).T
# toronto_profiles = pd.read_csv('Neighbourhoods_transpose_b_UTF8.csv', index_col=1)
toronto_profiles = pd.read_csv('Neighbourhoods_transpose_UTF8.csv', index_col=0)
In [59]:
toronto_profiles.columns
Out[59]:
In [60]:
toronto_profiles.shape
Out[60]:
In [61]:
toronto_profiles.head(10)
Out[61]:
In [62]:
# df.sort_values(by=['col1'])
toronto_profiles_nn_sort = toronto_profiles.sort_values(by=['NeighbourhoodNumber'])
toronto_profiles_nn_sort.head(10)
Out[62]:
Change Population Density string data with ',' to type integer¶
In [63]:
# PopulationDensity_per_square_km is a String with a , to represent numbers over 999.
# Change to int
toronto_profiles_nn_sort.dtypes
Out[63]:
In [64]:
df = toronto_profiles_nn_sort.copy()
In [65]:
df['PopulationDensity_per_square_km'] = df['PopulationDensity_per_square_km'].replace({',': ''}, regex=True).astype(int)
df.dtypes
Out[65]:
In [66]:
df.head(7)
Out[66]:
In [62]:
# df.to_pickle(file_name)
# where to save it, usually as a .pkl
# df = pd.read_pickle(file_name)
df.to_pickle('neighbourhood_pop_density_20200629.pkl')
In [ ]:
In [67]:
# neighborhoods_geojson = Neighbourhoods.geojson
# https://open.toronto.ca/dataset/neighbourhoods/
# https://github.com/jasonicarter/toronto-geojson/blob/master/toronto_crs84.geojson
# Boundaries of City of Toronto Neighbourhoods.
# https://ckan0.cf.opendata.inter.prod-toronto.ca/dataset/neighbourhoods/resource/a083c865-6d60-4d1d-b6c6-b0c8a85f9c15?view_id=204f29d7-2984-4e81-baed-3f5ed0582b8d
# http://www.arcgis.com/home/webmap/viewer.html?url=https://services.arcgis.com/As5CFN3ThbQpy8Ph/ArcGIS/rest/services/Toronto_Neighbourhoods/FeatureServer/0&source=sd
In [68]:
# Toronto City Hall 43.653908, -79.384293
# Both GeoJSON and TopoJSON layers can be passed to the map as an overlay, and multiple layers can be visualized on the same map
# 'r' Open for reading (default)
neighborhood_boundaries = r'Neighbourhoods.geojson'
map_toronto_neigh = folium.Map(location=[43.653908, -79.384293], zoom_start=11)
folium.GeoJson(
neighborhood_boundaries,
name='geojson'
).add_to(map_toronto_neigh)
folium.LayerControl().add_to(map_toronto_neigh)
map_toronto_neigh
Out[68]:
Map of Toronto with geojson overlay showing all neighbourhood boundaries.¶
In [ ]:
In [69]:
neighborhood_boundaries
Out[69]:
In [71]:
with open('Neighbourhoods.geojson') as f:
boundaries = geojson.load(f)
features = boundaries['features'][0]
In [72]:
neigh_code = boundaries['features'][0]['properties']
type(neigh_code)
Out[72]:
In [73]:
neigh_code.keys()
Out[73]:
In [74]:
# print(Dict[1])
neigh_code['AREA_SHORT_CODE']
Out[74]:
In [75]:
boundaries['features'][0]['properties']
Out[75]:
In [76]:
# Access single neighbourhood code for first entry
boundaries['features'][0]['properties']['AREA_SHORT_CODE']
Out[76]:
In [ ]:
In [69]:
# Find quantiles for color scale.
print('max: ', df.PopulationDensity_per_square_km.max())
print('75%: ', df.PopulationDensity_per_square_km.quantile(.75))
print('60%: ', df.PopulationDensity_per_square_km.quantile(.60))
print('45%: ', df.PopulationDensity_per_square_km.quantile(.45))
print('30%: ', df.PopulationDensity_per_square_km.quantile(.30))
print('15%: ', df.PopulationDensity_per_square_km.quantile(.15))
print('min: ', df.PopulationDensity_per_square_km.min())
In [ ]:
# population density choropleth map
# Population Density from 'df' dataframe. This is 'data' in folium.Choropleth
# Boundaries for neighbourhoods from file 'Neighbourhoods.geojson'
In [77]:
import json
# load geo_json
# shapefiles can be converted to geojson with QGIS
with open('Neighbourhoods.geojson') as f:
boundaries_geo = json.load(f)
# add feature 'id' county name to geojson
# access features
for i in boundaries_geo['features']:
i['id'] = i['properties']['AREA_SHORT_CODE']
new_boundaries_geo = boundaries_geo
# load data associated with geo_json
# map
map_choropleth = folium.Map(location=[43.653908, -79.384293], zoom_start=11)
# choropleth
folium.Choropleth(
geo_data=new_boundaries_geo,
name='choropleth',
data=df,
columns=['NeighbourhoodNumber','PopulationDensity_per_square_km'],
# see folium.Choropleth? for details on key_on
# key_on='feature.id',
key_on='feature.properties.AREA_SHORT_CODE',
fill_color='BuPu',
#threshold_scale=[0, 2, 4, 8, 16, 32],
#bins=9,
#bins=[1000, 2000, 3000, 4000, 5000, 7500, 10000, 15000, 45000],
bins=[1040, 2853, 3882, 4717, 6300, 7600, 45000],
fill_opacity=0.5,
line_opacity=0.5,
legend_name='Population',
highlight=True
).add_to(map_choropleth)
# layer control to turn choropleth on or off
folium.LayerControl().add_to(map_choropleth)
# display map
map_choropleth
Out[77]:
Choropleth map of neighborhoods color coded by population density¶
In [ ]:
In [ ]:
Foursquare Venue Data¶
In [ ]:
Define function getVenuesTopPicks¶
In [12]:
def getVenuesTopPicks(near, categories):
topPicks_all = pd.DataFrame(columns=['name', 'categories', 'address', 'lat', 'lng'])
i = 0
radius =20000
section = 'topPicks'
LIMIT = 50
openNow = 1
for categoryId in categories:
url = 'https://api.foursquare.com/v2/venues/explore?&client_id={}&client_secret={}&v={}&near={}&radius={}§ion={}&categoryId={}&limit={}&openNow={}'.format(
CLIENT_ID,
CLIENT_SECRET,
VERSION,
near,
radius,
section,
categoryId,
LIMIT,
openNow)
print(url) # display URL
# make the GET request
results = requests.get(url).json()
top_venues = results['response']['groups'][0]['items']
top_venues_df = pd.json_normalize(top_venues) # flatten JSON
# filter columns
filtered_columns = ['venue.name', 'venue.categories', 'venue.location.address', 'venue.location.lat', 'venue.location.lng']
nearby_top_venues = top_venues_df.loc[:, filtered_columns]
# filter the category for each row
nearby_top_venues['venue.categories'] = nearby_top_venues.apply(get_categories, axis=1)
# clean columns
nearby_top_venues.columns = [col.split(".")[-1] for col in nearby_top_venues.columns]
topPicks_all = pd.concat([topPicks_all, nearby_top_venues])
print('topPicks_all size: ', topPicks_all.shape)
i = i +1
topPicks_all = topPicks_all.drop_duplicates(subset='name', ignore_index=True)
print('topPicks_all size: ', topPicks_all.shape)
return(topPicks_all)
Run /venues/explore topPicks & pickle results dataframe¶
In [13]:
# getVenuesTopPicks function
# Run the above function
# Supply near as City, State, Country
# A string naming a place in the world. If the near string is not geocodable, returns a failed_geocode error.
# Otherwise, searches within the bounds of the geocode and adds a geocode object to the response.
near = 'North York, ON, Canada'
# supply a list of Categories to explore top picks for.
# Categories:
# College & U. 4d4b7105d754a06372d81259; Coffee Shop 4bf58dd8d48988d1e0931735; Dessert Shop 4bf58dd8d48988d1d0941735;
# Park 4bf58dd8d48988d163941735; Botanical Garden 52e81612bcbc57f1066b7a22'
# Cafe' 4bf58dd8d48988d16d941735
# categoryId = '4bf58dd8d48988d16d941735'
categories = ['4bf58dd8d48988d16d941735', '4bf58dd8d48988d1e0931735', '4bf58dd8d48988d1d0941735']
topPicks = getVenuesTopPicks(near, categories)
In [14]:
topPicks
Out[14]:
In [ ]:
Pickle time sensitive results data so that it can stored offline and later retrieved for comparison.¶
The pickle module implements binary protocols for serializing and de-serializing a Python object structure.¶
“Pickling” is the process whereby a Python object hierarchy is converted into a byte stream, and “unpickling” is the inverse operation,¶
whereby a byte stream (from a binary file or bytes-like object) is converted back into an object hierarchy.¶
Pickling (and unpickling) is alternatively known as “serialization”¶
In [15]:
# df.to_pickle(file_name) # where to save it, usually as a .pkl
# df = pd.read_pickle(file_name)
topPicks.to_pickle('north_york_coffee_cafe_dessert_topPicks_1210am_0702.pkl')
In [18]:
topPicks_7pm = pd.read_pickle('north_york_coffee_cafe_dessert_topPicks_720pm_0701.pkl')
In [19]:
topPicks_12am = topPicks
In [ ]:
Create a map to show TopPicks venues at 7pm compared to 12am to show to demonstate the "openNow" parameter.¶
In [20]:
# Midtown
# Toronto, ON, Canada
latitude = 43.698484
longitude = -79.397192
In [21]:
# parse_html=True
map_topPicks_openNow = folium.Map(location=[latitude, longitude], zoom_start=12, control_scale=True)
# I can add markers on the map with popup for Postal Code but borough name will not work reliably with variety of
# characters, such as / . ''. Use folium.Popup with parse_html
# Coffee Cafe Desserts OpenNow
for i in range(0,len(topPicks_7pm)):
folium.Marker([topPicks_7pm.iloc[i]['lat'], topPicks_7pm.iloc[i]['lng']],
popup=folium.Popup(topPicks_7pm.iloc[i]['name'], parse_html=True),
icon=folium.Icon(color='blue', icon='info-sign')
).add_to(map_topPicks_openNow)
# Coffee Cafe Desserts OpenNow
for i in range(0,len(topPicks)):
folium.Marker([topPicks.iloc[i]['lat'], topPicks.iloc[i]['lng']],
popup=folium.Popup(topPicks.iloc[i]['name'], parse_html=True),
icon=folium.Icon(color='darkpurple', icon='info-sign')
).add_to(map_topPicks_openNow)
# m.save('index.html')
map_topPicks_openNow.save('map_TopPicks_openNow_12am_20200702.html')
map_topPicks_openNow
Out[21]:
Top Picks for Cafe', Dessert & Coffee cluster around downtown Toronto¶
In [ ]:
In [ ]:
In [26]:
# df = pd.read_pickle(file_name)
top_Starbucks_11am = pd.read_pickle('north_york_top_Starbucks_11am_0701.pkl')
top_Starbucks_11am.head(10)
Out[26]:
In [ ]:
# map_TopPicks_day_latenight
In [30]:
# map_TopPicks_day_latenight
# parse_html=True
# Midtown
# Toronto, ON, Canada
latitude = 43.698484
longitude = -79.397192
map_topPicks = folium.Map(location=[latitude, longitude], zoom_start=12, control_scale=True)
# I can add markers on the map with popup for Postal Code but borough name will not work reliably with variety of
# characters, such as / . ''. Use folium.Popup with parse_html
# Coffee OpenNow
for i in range(0,len(top_Starbucks_11am)):
folium.Marker([top_Starbucks_11am.iloc[i]['lat'], top_Starbucks_11am.iloc[i]['lng']],
popup=folium.Popup(top_Starbucks_11am.iloc[i]['name'], parse_html=True),
icon=folium.Icon(color='orange', icon='info-sign')
).add_to(map_topPicks)
# Coffee Cafe Desserts OpenNow
for i in range(0,len(topPicks_7pm)):
folium.Marker([topPicks_7pm.iloc[i]['lat'], topPicks_7pm.iloc[i]['lng']],
popup=folium.Popup(topPicks_7pm.iloc[i]['name'], parse_html=True),
icon=folium.Icon(color='blue', icon='info-sign')
).add_to(map_topPicks)
# Coffee Cafe Desserts OpenNow
for i in range(0,len(topPicks)):
folium.Marker([topPicks.iloc[i]['lat'], topPicks.iloc[i]['lng']],
popup=folium.Popup(topPicks.iloc[i]['name'], parse_html=True),
icon=folium.Icon(color='darkpurple', icon='info-sign')
).add_to(map_topPicks)
# m.save('index.html')
# map_topPicks_openNow.save('map_TopPicks_openNow_730pm_20200701.html')
map_topPicks
Out[30]:
In [ ]:
Venues Search¶
Foursquare /venues/search will produce all results that match query term and location.¶
Venues Search will not be affected by time of day and show all matching results.¶
In [ ]:
In [25]:
# Run a Venue Search with Query for Starbucks
# Toronto, ON, Canada-
#1 Blue Jays Way
# Toronto, ON M5V 1J3, Canada
# 43.641853, -79.389716
# City Hall
# 43.653908, -79.384293
# Scarborough
# 43.777008, -79.224105
# 100 Scarlett Rd
# York, ON M6N 4K2, Canada
# 43.670158, -79.507366
# North York Centre
# Starbucks 5140 Yonge St, North York, ON M2N 6L7, Canada
# 43.768353, -79.413038
# Add East York
# Danforth Village
# Toronto, ON, Canada
# 43.687972, -79.301815
In [ ]:
In [26]:
# Coordinates to use for 4 Regions around Toronto with 15 km search radius each,
# [[ 43.777008, -79.224105] East, Scarborough
# [ 43.768633, -79.413312 ] North York
# [ 43.653908, -79.384293] Downtown Toronto
# 43.670158, -79.507366 West, York, Etobicoke, ON
# 43.687972, -79.301815 East York
coordinates = np.array([43.777008, -79.224105, 43.768633, -79.413312, 43.653908, -79.384293, 43.670158, -79.507366, 43.687972, -79.301815 ]).reshape(5, 2)
coordinates
Out[26]:
Function to create Foursquare API Request¶
In [27]:
def getVenuesSearch(query, latitudes, longitudes, radius = 15000, LIMIT = 50):
search_results_all = pd.DataFrame(columns=['id','address', 'lat', 'lng'])
i = 0
for latitude, longitude in zip(latitudes, longitudes):
print('centroid: ', i, latitude, longitude)
# create the API request URL
url = 'https://api.foursquare.com/v2/venues/search?&client_id={}&client_secret={}&v={}&ll={},{}&radius={}&limit={}&query={}'.format(
CLIENT_ID,
CLIENT_SECRET,
VERSION,
latitude,
longitude,
radius,
LIMIT,
query)
print(url) # display URL
# make the GET request
results = requests.get(url).json()
search_results = results['response']['venues']
search_results_df = pd.json_normalize(search_results) # flatten JSON
# filter columns
filtered_columns = ['id', 'location.address', 'location.lat', 'location.lng']
nearby_search_venues = search_results_df.loc[:, filtered_columns]
# clean columns
nearby_search_venues.columns = [col.split(".")[-1] for col in nearby_search_venues.columns]
search_results_all = pd.concat([search_results_all, nearby_search_venues])
print('search_results_all size: ', search_results_all.shape)
i = i +1
search_results_all = search_results_all.drop_duplicates(subset='id', ignore_index=True)
print('search_results_all size: ', search_results_all.shape)
return(search_results_all)
In [121]:
# getVenuesSearch function
# Run the above function
# Supply venue name query string - Example 'Starbucks'
# Supply numpy array of latitude & longitude values
query = 'Starbucks'
all_starbucks = getVenuesSearch(query, latitudes=coordinates[:, 0], longitudes=coordinates[:, 1])
In [122]:
all_starbucks.head(20)
Out[122]:
In [123]:
# Map of All Starbucks locations around Toronto
all_starbucks.shape
Out[123]:
In [124]:
#1 Blue Jays Way
# Toronto, ON M5V 1J3, Canada
# 43.641853, -79.389716
latitude = 43.641853
longitude = -79.389716
In [125]:
# parse_html=True
All_Starbucks_Map = folium.Map(location=[latitude, longitude], zoom_start=12, control_scale=True)
# I can add markers on the map with popup for Postal Code but borough name will not work reliably with variety of
# characters, such as / . ''. Use folium.Popup with parse_html
# Starbucks Green
for i in range(0,len(all_starbucks)):
folium.Marker([all_starbucks.iloc[i]['lat'], all_starbucks.iloc[i]['lng']],
popup=folium.Popup(all_starbucks.iloc[i]['address'], parse_html=True),
icon=folium.Icon(color='green', icon='info-sign')
).add_to(All_Starbucks_Map)
#map.save('map_All_Starbucks_15km_20200629.html')
All_Starbucks_Map.save('map_All_Starbucks_15km_20200629.html')
All_Starbucks_Map
Out[125]:
Map of All 160 Starbucks locations around Toronto¶
In [ ]:
In [ ]:
# k-means_clustering
K-Means Clustering¶
In [126]:
all_starbucks.columns
Out[126]:
In [127]:
filtered_columns = ['lat', 'lng']
AllStar_lat_lng_df = all_starbucks.loc[:, filtered_columns]
AllStar_lat_lng_df.head(10)
Out[127]:
In [128]:
# Convert All_Toronto_Starbucks_df to numpy array
AllStar = AllStar_lat_lng_df.to_numpy(copy=True)
AllStar
Out[128]:
In [129]:
AllStar.shape
Out[129]:
In [130]:
kmeans = KMeans(n_clusters=4)
kmeans.fit(AllStar)
y_kmeans = kmeans.predict(AllStar)
In [136]:
# Latitude - equator=0 to poles=90 (Array 0, first value in coordinate) is represented on graph as Y axis
# Longitude - (Array 1 values, 2nd in pair ) East West, Greenwich, England, is defined as 0° longitude with west longitudes being negative. X axis on graph plot.
plt.figure(figsize=(16,9))
plt.scatter(AllStar[:, 1], AllStar[:, 0], c=y_kmeans, s=50, cmap='viridis')
centers = kmeans.cluster_centers_
plt.scatter(centers[:, 1], centers[:, 0], c='black', s=200, alpha=0.5);
plt.colorbar(); # show color scale
#plt.gca().invert_yaxis()
In [132]:
print(centers)
In [133]:
y_kmeans
Out[133]:
In [ ]:
Predict cluster label for a new coordinate¶
In [134]:
# Where I want to go:
# Starbucks Reserve Bar
# 1090 Don Mills Rd (at Lawrence Ave)
# Toronto ON M3C 0H1
# Canada
third_test = np.array([43.735977, -79.344265])
third_test = third_test.reshape(1, -1)
third_test_label = kmeans.predict(third_test)
third_test_label
Out[134]:
In [ ]:
# Starbucks Reserve Bar is in Cluster 2
In [137]:
plt.figure(figsize=(16,9))
plt.scatter(AllStar[:, 1], AllStar[:, 0], c=y_kmeans, s=50, cmap='viridis')
centers = kmeans.cluster_centers_
plt.scatter(centers[:, 1], centers[:, 0], c='black', s=200, alpha=0.5)
plt.scatter(third_test[:, 1], third_test[:, 0], c=third_test_label, s=750, cmap='viridis', alpha=0.25)
plt.colorbar(); # show color scale
In [ ]:
# Starbucks Reserve Bar highlighted with larger diameter, .25 alpha circle marker.
Now Display All Starbucks locations color coded by K-Means Cluster¶
In [138]:
kcolors = ['green', 'orange', 'purple', 'darkblue']
In [139]:
kcolors
Out[139]:
In [ ]:
In [145]:
# parse_html=True
latitude = 43.735977
longitude = -79.344265
All_Starbucks_Map = folium.Map(location=[latitude, longitude], zoom_start=11, control_scale=True)
# I can add markers on the map with popup for Postal Code but borough name will not work reliably with variety of
# characters, such as / . ''. Use folium.Popup with parse_html
# Starbucks Green
for i in range(0,len(all_starbucks)):
folium.Marker([all_starbucks.iloc[i]['lat'], all_starbucks.iloc[i]['lng']],
popup=folium.Popup(all_starbucks.iloc[i]['address'], parse_html=True),
icon=folium.Icon(color=kcolors[y_kmeans[i]], icon='info-sign')
).add_to(All_Starbucks_Map)
# y_kmeans markers for cluster centers
for i in range(0, len(centers)):
folium.Marker([centers[i, 0], centers[i, 1]],
icon=folium.Icon(color='black', icon='info-sign')
).add_to(All_Starbucks_Map)
folium.Marker([third_test[:, 0], third_test[:, 1]],
popup=folium.Popup('Starbucks Reserve Bar'),
icon=folium.Icon(color='darkpurple', icon='info-sign')
).add_to(All_Starbucks_Map)
All_Starbucks_Map.save('map_All_Starbucks_Clusters_20200629.html')
All_Starbucks_Map
Out[145]:
Starbucks locations color coded by K-means cluster it belongs to. Starbucks Reserve Bar icon in dark purple.¶
In [ ]:
In [ ]:
In [148]:
import json
# load geo_json
# shapefiles can be converted to geojson with QGIS
with open('Neighbourhoods.geojson') as f:
boundaries_geo = json.load(f)
# add feature 'id' county name to geojson
# access features
for i in boundaries_geo['features']:
i['id'] = i['properties']['AREA_SHORT_CODE']
new_boundaries_geo = boundaries_geo
# load data associated with geo_json
# map 43.735977, -79.344265
map_AllStar_choropleth = folium.Map(location=[43.735977, -79.344265], zoom_start=12)
# choropleth
folium.Choropleth(
geo_data=new_boundaries_geo,
name='choropleth',
data=df,
columns=['NeighbourhoodNumber','PopulationDensity_per_square_km'],
# see folium.Choropleth? for details on key_on
# key_on='feature.id',
key_on='feature.properties.AREA_SHORT_CODE',
fill_color='BuPu',
#threshold_scale=[0, 2, 4, 8, 16, 32],
#bins=9,
#bins=[1000, 2000, 3000, 4000, 5000, 7500, 10000, 15000, 45000],
bins=[1040, 2853, 3882, 4717, 6300, 7600, 45000],
fill_opacity=0.5,
line_opacity=0.5,
legend_name='Population',
highlight=True
).add_to(map_AllStar_choropleth)
# layer control to turn choropleth on or off
folium.LayerControl().add_to(map_AllStar_choropleth)
# Starbucks Green
for i in range(0,len(all_starbucks)):
folium.Marker([all_starbucks.iloc[i]['lat'], all_starbucks.iloc[i]['lng']],
popup=folium.Popup(all_starbucks.iloc[i]['address'], parse_html=True),
icon=folium.Icon(color=kcolors[y_kmeans[i]], icon='info-sign')
).add_to(map_AllStar_choropleth)
# y_kmeans
for i in range(0, len(centers)):
folium.Marker([centers[i, 0], centers[i, 1]],
icon=folium.Icon(color='black', icon='info-sign')
).add_to(map_AllStar_choropleth)
# display map
map_AllStar_choropleth.save('map_All_Starbucks_Clusters_PopDensity_20200629.html')
map_AllStar_choropleth
Out[148]:
Starbucks K-Means Clusters marked on Population Density Map for Toronto's Neighbourhoods¶
In [ ]:
In [ ]:
In [ ]:
In [ ]: